<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>36.7. Extensibility</title>
<link rel="stylesheet" href="dbstyle.css" type="text/css">
<meta name="generator" content="DocBook XSL Stylesheets V1.72.0">
<link rel="start" href="index.html" title="Programmer's Reference Guide">
<link rel="up" href="zend.search.lucene.html" title="Chapter 36. Zend_Search_Lucene">
<link rel="prev" href="zend.search.lucene.charset.html" title="36.6. Character Set">
<link rel="next" href="zend.search.lucene.java-lucene.html" title="36.8. Interoperating with Java Lucene">
<link rel="chapter" href="introduction.html" title="Chapter 1. Introduction to Zend Framework">
<link rel="chapter" href="zend.acl.html" title="Chapter 2. Zend_Acl">
<link rel="chapter" href="zend.auth.html" title="Chapter 3. Zend_Auth">
<link rel="chapter" href="zend.cache.html" title="Chapter 4. Zend_Cache">
<link rel="chapter" href="zend.config.html" title="Chapter 5. Zend_Config">
<link rel="chapter" href="zend.console.getopt.html" title="Chapter 6. Zend_Console_Getopt">
<link rel="chapter" href="zend.controller.html" title="Chapter 7. Zend_Controller">
<link rel="chapter" href="zend.currency.html" title="Chapter 8. Zend_Currency">
<link rel="chapter" href="zend.date.html" title="Chapter 9. Zend_Date">
<link rel="chapter" href="zend.db.html" title="Chapter 10. Zend_Db">
<link rel="chapter" href="zend.debug.html" title="Chapter 11. Zend_Debug">
<link rel="chapter" href="zend.dojo.html" title="Chapter 12. Zend_Dojo">
<link rel="chapter" href="zend.dom.html" title="Chapter 13. Zend_Dom">
<link rel="chapter" href="zend.exception.html" title="Chapter 14. Zend_Exception">
<link rel="chapter" href="zend.feed.html" title="Chapter 15. Zend_Feed">
<link rel="chapter" href="zend.filter.html" title="Chapter 16. Zend_Filter">
<link rel="chapter" href="zend.form.html" title="Chapter 17. Zend_Form">
<link rel="chapter" href="zend.gdata.html" title="Chapter 18. Zend_Gdata">
<link rel="chapter" href="zend.http.html" title="Chapter 19. Zend_Http">
<link rel="chapter" href="zend.infocard.html" title="Chapter 20. Zend_InfoCard">
<link rel="chapter" href="zend.json.html" title="Chapter 21. Zend_Json">
<link rel="chapter" href="zend.layout.html" title="Chapter 22. Zend_Layout">
<link rel="chapter" href="zend.ldap.html" title="Chapter 23. Zend_Ldap">
<link rel="chapter" href="zend.loader.html" title="Chapter 24. Zend_Loader">
<link rel="chapter" href="zend.locale.html" title="Chapter 25. Zend_Locale">
<link rel="chapter" href="zend.log.html" title="Chapter 26. Zend_Log">
<link rel="chapter" href="zend.mail.html" title="Chapter 27. Zend_Mail">
<link rel="chapter" href="zend.measure.html" title="Chapter 28. Zend_Measure">
<link rel="chapter" href="zend.memory.html" title="Chapter 29. Zend_Memory">
<link rel="chapter" href="zend.mime.html" title="Chapter 30. Zend_Mime">
<link rel="chapter" href="zend.openid.html" title="Chapter 31. Zend_OpenId">
<link rel="chapter" href="zend.paginator.html" title="Chapter 32. Zend_Paginator">
<link rel="chapter" href="zend.pdf.html" title="Chapter 33. Zend_Pdf">
<link rel="chapter" href="zend.registry.html" title="Chapter 34. Zend_Registry">
<link rel="chapter" href="zend.rest.html" title="Chapter 35. Zend_Rest">
<link rel="chapter" href="zend.search.lucene.html" title="Chapter 36. Zend_Search_Lucene">
<link rel="chapter" href="zend.server.html" title="Chapter 37. Zend_Server">
<link rel="chapter" href="zend.service.html" title="Chapter 38. Zend_Service">
<link rel="chapter" href="zend.session.html" title="Chapter 39. Zend_Session">
<link rel="chapter" href="zend.soap.html" title="Chapter 40. Zend_Soap">
<link rel="chapter" href="zend.test.html" title="Chapter 41. Zend_Test">
<link rel="chapter" href="zend.text.html" title="Chapter 42. Zend_Text">
<link rel="chapter" href="zend.timesync.html" title="Chapter 43. Zend_TimeSync">
<link rel="chapter" href="zend.translate.html" title="Chapter 44. Zend_Translate">
<link rel="chapter" href="zend.uri.html" title="Chapter 45. Zend_Uri">
<link rel="chapter" href="zend.validate.html" title="Chapter 46. Zend_Validate">
<link rel="chapter" href="zend.version.html" title="Chapter 47. Zend_Version">
<link rel="chapter" href="zend.view.html" title="Chapter 48. Zend_View">
<link rel="chapter" href="zend.xmlrpc.html" title="Chapter 49. Zend_XmlRpc">
<link rel="appendix" href="requirements.html" title="Appendix A. Zend Framework Requirements">
<link rel="appendix" href="coding-standard.html" title="Appendix B. Zend Framework Coding Standard for PHP">
<link rel="appendix" href="copyrights.html" title="Appendix C. Copyright Information">
<link rel="index" href="the.index.html" title="Index">
<link rel="subsection" href="zend.search.lucene.extending.html#zend.search.lucene.extending.analysis" title="36.7.1. Text Analysis">
<link rel="subsection" href="zend.search.lucene.extending.html#zend.search.lucene.extending.filters" title="36.7.2. Tokens Filtering">
<link rel="subsection" href="zend.search.lucene.extending.html#zend.search.lucene.extending.scoring" title="36.7.3. Scoring Algorithms">
<link rel="subsection" href="zend.search.lucene.extending.html#zend.search.lucene.extending.storage" title="36.7.4. Storage Containers">
</head>
<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
<div class="navheader"><table width="100%" summary="Navigation header">
<tr><th colspan="3" align="center">36.7. Extensibility</th></tr>
<tr>
<td width="20%" align="left">
<a accesskey="p" href="zend.search.lucene.charset.html">Prev</a> </td>
<th width="60%" align="center">Chapter 36. Zend_Search_Lucene</th>
<td width="20%" align="right"> <a accesskey="n" href="zend.search.lucene.java-lucene.html">Next</a>
</td>
</tr>
</table></div>
<div class="sect1" lang="en">
<div class="titlepage"><div><div><h2 class="title" style="clear: both">
<a name="zend.search.lucene.extending"></a>36.7. Extensibility</h2></div></div></div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.extending.analysis"></a>36.7.1. Text Analysis</h3></div></div></div>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_Analyzer</code> class is used by the indexer to tokenize document
            text fields.
        </p>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_Analyzer::getDefault()</code> and <code class="code">
            Zend_Search_Lucene_Analysis_Analyzer::setDefault()</code> methods are used
            to get and set the default analyzer.
        </p>
<p>
            You can assign your own text analyzer or choose it from the set of predefined analyzers:
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Text</code> and
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive</code> (default).
            Both of them interpret tokens as sequences of letters.
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive</code> converts all tokens
            to lower case.
        </p>
<p>
            To switch between analyzers:
        </p>
<pre class="programlisting">&lt;?php
Zend_Search_Lucene_Analysis_Analyzer::setDefault(
    new Zend_Search_Lucene_Analysis_Analyzer_Common_Text());
...
$index-&gt;addDocument($doc);
        </pre>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common</code> class is designed to be an ancestor of all user
            defined analyzers. User should only define the <code class="code">reset()</code> and <code class="code">nextToken()</code> methods,
            which takes its string from the $_input member and returns tokens one by one
            (a <code class="code">null</code> value indicates the end of the stream).
        </p>
<p>
            The <code class="code">nextToken()</code> method should call the <code class="code">normalize()</code> method on each
            token. This will allow you to use token filters with your analyzer.
        </p>
<p>
            Here is an example of a custom analyzer, which accepts words with digits as terms:

            </p>
<div class="example">
<a name="zend.search.lucene.extending.analysis.example-1"></a><p class="title"><b>Example 36.1. Custom text Analyzer.</b></p>
<div class="example-contents"><pre class="programlisting">&lt;?php
/** Here is a custom text analyser, which treats words with digits as one term */


/** Zend_Search_Lucene_Analysis_Analyzer hierarchy */
require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';

class My_Analyzer extends Zend_Search_Lucene_Analysis_Analyzer_Common
{
    private $_position;

    /**
     * Reset token stream
     */
    public function reset()
    {
        $this-&gt;_position = 0;
    }

    /**
     * Tokenization stream API
     * Get next token
     * Returns null at the end of stream
     *
     * @return Zend_Search_Lucene_Analysis_Token|null
     */
    public function nextToken()
    {
        if ($this-&gt;_input === null) {
            return null;
        }

        while ($this-&gt;_position &lt; strlen($this-&gt;_input)) {
            // skip white space
            while ($this-&gt;_position &lt; strlen($this-&gt;_input) &amp;&amp;
                   !ctype_alnum( $this-&gt;_input[$this-&gt;_position] )) {
                $this-&gt;_position++;
            }

            $termStartPosition = $this-&gt;_position;

            // read token
            while ($this-&gt;_position &lt; strlen($this-&gt;_input) &amp;&amp;
                   ctype_alnum( $this-&gt;_input[$this-&gt;_position] )) {
                $this-&gt;_position++;
            }

            // Empty token, end of stream.
            if ($this-&gt;_position == $termStartPosition) {
                return null;
            }

            $token = new Zend_Search_Lucene_Analysis_Token(
                                      substr($this-&gt;_input,
                                             $termStartPosition,
                                             $this-&gt;_position - $termStartPosition),
                                      $termStartPosition,
                                      $this-&gt;_position);
            $token = $this-&gt;normalize($token);
            if ($token !== null) {
                return $token;
            }
            // Continue if token is skipped
        }

        return null;
    }
}

Zend_Search_Lucene_Analysis_Analyzer::setDefault(
    new My_Analyzer());
                </pre></div>
</div>
<p><br class="example-break">
        </p>
</div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.extending.filters"></a>36.7.2. Tokens Filtering</h3></div></div></div>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common</code> analyzer also offers a token filtering
            mechanism.
        </p>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_TokenFilter</code> class provides an abstract interface for such filters.
            Your own filters should extend this class either directly or indirectly.
        </p>
<p>
            Any custom filter must implement the <code class="code">normalize()</code> method which may transform input token or signal that
            the current token should be skipped.
        </p>
<p>
            There are three filters already defined in the analysis subpackage:
            </p>
<div class="itemizedlist"><ul type="disc">
<li><p>
                        <code class="code">Zend_Search_Lucene_Analysis_TokenFilter_LowerCase</code>
                    </p></li>
<li><p>
                        <code class="code">Zend_Search_Lucene_Analysis_TokenFilter_ShortWords</code>
                    </p></li>
<li><p>
                        <code class="code">Zend_Search_Lucene_Analysis_TokenFilter_StopWords</code>
                    </p></li>
</ul></div>
<p>
        </p>
<p>
            The <code class="code">LowerCase</code> filter is already used for
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive</code> analyzer
            by default.
        </p>
<p>
            The <code class="code">ShortWords</code> and <code class="code">StopWords</code> filters may be used with pre-defined or custom
            analyzers like this:
            </p>
<pre class="programlisting">&lt;?php
$stopWords = array('a', 'an', 'at', 'the', 'and', 'or', 'is', 'am');
$stopWordsFilter = new Zend_Search_Lucene_Analysis_TokenFilter_StopWords($stopWords);

$analyzer = new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive();
$analyzer-&gt;addFilter($stopWordsFilter);

Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);
            </pre>
<p>
            </p>
<pre class="programlisting">&lt;?php
$shortWordsFilter = new Zend_Search_Lucene_Analysis_TokenFilter_ShortWords();

$analyzer = new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive();
$analyzer-&gt;addFilter($shortWordsFilter);

Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);
            </pre>
<p>
        </p>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_TokenFilter_StopWords</code> constructor takes an array of stop-words
            as an input. But stop-words may be also loaded from a file:
            </p>
<pre class="programlisting">&lt;?php
$stopWordsFilter = new Zend_Search_Lucene_Analysis_TokenFilter_StopWords();
$stopWordsFilter-&gt;loadFromFile($my_stopwords_file);

$analyzer = new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive();
$analyzer-&gt;addFilter($stopWordsFilter);

Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer);
            </pre>
<p>
            This file should be a common text file with one word in each line. The '#' character marks a line as a comment.
        </p>
<p>
            The <code class="code">Zend_Search_Lucene_Analysis_TokenFilter_ShortWords</code> constructor has one optional argument.
            This is the word length limit, set by default to 2.
        </p>
</div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.extending.scoring"></a>36.7.3. Scoring Algorithms</h3></div></div></div>
<p>
            The score of a document <code class="literal">d</code> for a query <code class="literal">q</code> 
            is defined as follows:
        </p>
<p>
            <code class="code">score(q,d) = sum( tf(t in d) * idf(t) * getBoost(t.field in d) * lengthNorm(t.field in d)  ) *
            coord(q,d) * queryNorm(q)</code>
        </p>
<p>
            tf(t in d) - <code class="code">Zend_Search_Lucene_Search_Similarity::tf($freq)</code> - a score factor based on the frequency of a term or phrase in a document.
        </p>
<p>
            idf(t) - <code class="code">Zend_Search_Lucene_Search_Similarity::tf($term, $reader)</code> - a score factor for a simple term with the specified index.
        </p>
<p>
            getBoost(t.field in d) - the boost factor for the term field.
        </p>
<p>
            lengthNorm($term) - the normalization value for a field given the total
            number of terms contained in a field. This value is stored within the index.
            These values, together with field boosts, are stored in an index and multiplied
            into scores for hits on each field by the search code.
        </p>
<p>
            Matches in longer fields are less precise, so implementations of this method
            usually return smaller values when numTokens is large, and larger values when numTokens is small.
        </p>
<p>
            coord(q,d) - <code class="code">Zend_Search_Lucene_Search_Similarity::coord($overlap, $maxOverlap)</code> - a
            score factor based on the fraction of all query terms that a document contains.
        </p>
<p>
            The presence of a large portion of the query terms indicates a better match
            with the query, so implementations of this method usually return larger values
            when the ratio between these parameters is large and smaller values when
            the ratio between them is small.
        </p>
<p>
            queryNorm(q) -  the normalization value for a query given the sum of the squared weights
            of each of the query terms. This value is then multiplied into the weight of each query
            term.
        </p>
<p>
            This does not affect ranking, but rather just attempts to make scores from different
            queries comparable.
        </p>
<p>
            The scoring algorithm can be customized by defining your own Similarity class. To do this
            extend the Zend_Search_Lucene_Search_Similarity class as defined below, then use
            the <code class="code">Zend_Search_Lucene_Search_Similarity::setDefault($similarity);</code> method to set it as default.
        </p>
<pre class="programlisting">&lt;?php
class MySimilarity extends Zend_Search_Lucene_Search_Similarity {
    public function lengthNorm($fieldName, $numTerms) {
        return 1.0/sqrt($numTerms);
    }

    public function queryNorm($sumOfSquaredWeights) {
        return 1.0/sqrt($sumOfSquaredWeights);
    }

    public function tf($freq) {
        return sqrt($freq);
    }

    /**
     * It's not used now. Computes the amount of a sloppy phrase match,
     * based on an edit distance.
     */
    public function sloppyFreq($distance) {
        return 1.0;
    }

    public function idfFreq($docFreq, $numDocs) {
        return log($numDocs/(float)($docFreq+1)) + 1.0;
    }

    public function coord($overlap, $maxOverlap) {
        return $overlap/(float)$maxOverlap;
    }
}

$mySimilarity = new MySimilarity();
Zend_Search_Lucene_Search_Similarity::setDefault($mySimilarity);
        </pre>
</div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.extending.storage"></a>36.7.4. Storage Containers</h3></div></div></div>
<p>
        The abstract class <code class="code">Zend_Search_Lucene_Storage_Directory</code> defines directory functionality.
        </p>
<p>
        The <code class="code">Zend_Search_Lucene</code> constructor uses either a string or a
        <code class="code">Zend_Search_Lucene_Storage_Directory</code> object
        as an input.
        </p>
<p>
        The <code class="code">Zend_Search_Lucene_Storage_Directory_Filesystem</code> class implements directory
        functionality for a file system.
        </p>
<p>
        If a string is used as an input for the <code class="code">Zend_Search_Lucene</code> constructor, then the index
        reader (<code class="code">Zend_Search_Lucene</code> object) treats it as a file system path and instantiates
        the <code class="code">Zend_Search_Lucene_Storage_Directory_Filesystem</code> object.
        </p>
<p>
        You can define your own directory implementation by extending the
        <code class="code">Zend_Search_Lucene_Storage_Directory</code> class.
        </p>
<p>
        <code class="code">Zend_Search_Lucene_Storage_Directory</code> methods:
        </p>
<pre class="programlisting">&lt;?php
abstract class Zend_Search_Lucene_Storage_Directory {
/**
 * Closes the store.
 *
 * @return void
 */
abstract function close();


/**
 * Creates a new, empty file in the directory with the given $filename.
 *
 * @param string $name
 * @return void
 */
abstract function createFile($filename);


/**
 * Removes an existing $filename in the directory.
 *
 * @param string $filename
 * @return void
 */
abstract function deleteFile($filename);


/**
 * Returns true if a file with the given $filename exists.
 *
 * @param string $filename
 * @return boolean
 */
abstract function fileExists($filename);


/**
 * Returns the length of a $filename in the directory.
 *
 * @param string $filename
 * @return integer
 */
abstract function fileLength($filename);


/**
 * Returns the UNIX timestamp $filename was last modified.
 *
 * @param string $filename
 * @return integer
 */
abstract function fileModified($filename);


/**
 * Renames an existing file in the directory.
 *
 * @param string $from
 * @param string $to
 * @return void
 */
abstract function renameFile($from, $to);


/**
 * Sets the modified time of $filename to now.
 *
 * @param string $filename
 * @return void
 */
abstract function touchFile($filename);


/**
 * Returns a Zend_Search_Lucene_Storage_File object for a given $filename in the directory.
 *
 * @param string $filename
 * @return Zend_Search_Lucene_Storage_File
 */
abstract function getFileObject($filename);

}
        </pre>
<p>
        The <code class="code">getFileObject($filename)</code> method of a <code class="code">Zend_Search_Lucene_Storage_Directory</code>
        instance returns a <code class="code">Zend_Search_Lucene_Storage_File</code> object.
        </p>
<p>
        The <code class="code">Zend_Search_Lucene_Storage_File</code> abstract class implements file abstraction and index
        file reading primitives.
        </p>
<p>
        You must also extend <code class="code">Zend_Search_Lucene_Storage_File</code> for your directory implementation.
        </p>
<p>
        Only two methods of <code class="code">Zend_Search_Lucene_Storage_File</code> must be overridden in your
        implementation:
        </p>
<pre class="programlisting">&lt;?php
class MyFile extends Zend_Search_Lucene_Storage_File {
    /**
     * Sets the file position indicator and advances the file pointer.
     * The new position, measured in bytes from the beginning of the file,
     * is obtained by adding offset to the position specified by whence,
     * whose values are defined as follows:
     * SEEK_SET - Set position equal to offset bytes.
     * SEEK_CUR - Set position to current location plus offset.
     * SEEK_END - Set position to end-of-file plus offset. (To move to
     * a position before the end-of-file, you need to pass a negative value
     * in offset.)
     * Upon success, returns 0; otherwise, returns -1
     *
     * @param integer $offset
     * @param integer $whence
     * @return integer
     */
    public function seek($offset, $whence=SEEK_SET) {
        ...
    }

    /**
     * Read a $length bytes from the file and advance the file pointer.
     *
     * @param integer $length
     * @return string
     */
    protected function _fread($length=1) {
        ...
    }
}
        </pre>
</div>
</div>
<div class="navfooter"><table width="100%" summary="Navigation footer">
<tr>
<td width="40%" align="left">
<a accesskey="p" href="zend.search.lucene.charset.html">Prev</a> </td>
<td width="20%" align="center"><a accesskey="u" href="zend.search.lucene.html">Up</a></td>
<td width="40%" align="right"> <a accesskey="n" href="zend.search.lucene.java-lucene.html">Next</a>
</td>
</tr>
<tr>
<td width="40%" align="left" valign="top">36.6. Character Set </td>
<td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td>
<td width="40%" align="right" valign="top"> 36.8. Interoperating with Java Lucene</td>
</tr>
</table></div>
<div class="revinfo"></div>
</body>
</html>
